<?php
require 'phpQuery.php';
/**
 * 多线程关键字抓取类 ...
 * @author zhutl
 * @example
 * $keywords="php";
 * $mp = new MultiHttpRequest($keywords);
 * $mp->set_page(3);
 * $mp->start();
 */
class SearchBook {
	public $urls = array();
	//抓取地址
	public $keywords = "";
	//关键字
	public $curlopt_header = 1;
	public $method = "POST";
	public $timeout = "";
	//超时时间
	public $page = 1;
	//采集页数
	public $type = "pdf";
	//采集文件类型
	public function __construct($keywords, $urls = array(), $timeout = 10) {
		//$this->urls = $urls;
		// array()
		$this -> keywords = $keywords;
		$this -> urls = $urls;
		//print_r($urls);exit;
		$this -> timeout = $timeout;
	}

	public function set_type($type) {
		$this -> type = $type;
		return $this;
	}

	public function set_keywords($keywords) {
		$this -> keywords = $keywords;
		return $this;
	}

	public function set_page($page) {
		$this -> page = $page;
		return $this;
	}

	public function set_timeout($timeout) {
		$this -> timeout = $timeout;
		return $this;

	}

	public function is_return_header($b) {
		$this -> curlopt_header = $b;
		return $this;
	}

	public function set_method($m) {
		$this -> medthod = strtoupper($m);
		return $this;
	}

	/** 获取格式化数据
	 * @return array();
	 */
	public function getDate() {

		$text = $this -> start();
		if(!empty($text)){
		$data = array();
		//print_r($text);exit;
		foreach ($text as $k => $p) {

			$data[$k] = $this -> dataFomart($p, $k);

		}
		//print_r($data);exit;

		$dd = $data;
		$m=0;
		foreach ($dd as $k1 => $p1) {
			foreach ($p1 as $p2) {
				$newdata[$m] = $p2;
				$m++;
			}
		}

		return $this -> unique_arr($newdata);
		
		}

	}

	/**获取第一次加载多数据
	 * 默认由百度获取
	 * @return array();
	 */
	public function getFristDate($keyword) {
		$keyword = urlencode($keyword);
		//百度抓取时候要吧URL转换
		$urls = array('http://www.baidu.com/s?wd=filetype%3Apdf%20' . $keyword . '&pn=0', 'http://www.baidu.com/s?wd=filetype%3Apdf%20' . $keyword . '&pn=10',
		'http://www.baidu.com/s?wd=filetype%3Apdf%20' . $keyword . '&pn=20'
		);
		//print_r($urls);
		$curl = $text = array();
		$handle = curl_multi_init();
		foreach ($urls as $k => $v) {
			//echo $v.'<br/>';
			$curl[$k] = $this -> add_handle($handle, $v);
		}
		$this -> exec_handle($handle);
		foreach ($urls as $k => $v) {
			$text[$k] = curl_multi_getcontent($curl[$k]);
			$text[$k] = mb_convert_encoding($text[$k] . "\n\n", "UTF-8", "GBK");
			curl_multi_remove_handle($handle, $curl[$k]);
		}
		curl_multi_close($handle);
		if(!empty($text)){
		$data = array();
		foreach ($text as $k => $p) {
			$data[$k] = $this -> bdFomart($p);
		}
		//$data=$this->bdFomart($text);
		//print_r($data);exit;
		$m = 0;
		$newdata = array();
		foreach ($data as $k1 => $p1) {
			foreach ($p1 as $p2) {
				$newdata[$m] = $p2;
				$m++;
			}
		}
		//print_r($newdata);exit;
		return $this -> unique_arr($newdata);
		}
	}

	private function bdFomart($text) {
		$text = str_replace("charset=gb2312", "charset=utf-8", $text);
		$text = preg_replace('/(<br.*?>)(.*?)(<br>)/', '\1<span class="remarks">\2</span>\3', $text);
		//echo $text;exit;
		phpQuery::newDocument($text);
		$artlist = pq('.f');
		$title = "";
		$url = "";
		$remarks = "";
		$start = 0;
		$data = array();
		foreach ($artlist as $li) {
			$url = pq($li) -> find('h3 a') -> attr("href");
			//print_r($url);
			if ($this -> type == end(explode(".", $url))) {
				//print_r(pq($li)->text());
				$title = pq($li) -> find('h3 a') -> text();
				$remarks = pq($li) -> find('span[class=remarks]') -> text();
				$data[$start] = array(title => $title, url => $url, source => 2, remarks => $remarks);
				$start++;

			}
		}
		return $data;
	}

	private function dataFomart($text, $p) {
		$k = $p % (count($this -> urls) / $this -> page);
		$pq = $this -> urls[$k]['pq'];
		$pq_url = $this -> urls[$k]['url'];
		$pq_title = $this -> urls[$k]['title'];
		$pq_remarks = $this -> urls[$k]['remarks'];
		$pq_remarks_replace = $this -> urls[$k]['remarks_replace'];
		$pq_remarks_replace_rule = $this -> urls[$k]['remarks_replace_rule'];
		$source = $this -> urls[$k]['source'];
		//print_r($this->urls[$k]);
		$text = str_replace("charset=gb2312", "charset=utf-8", $text);
		//echo $pq_remarks_replace;exit;
		if ($pq_remarks_replace) {
			$text = preg_replace($pq_remarks_replace_rule, $pq_remarks_replace, $text);
		}
		phpQuery::newDocument($text);
		$artlist = pq($pq);
		$title = "";
		$url = "";
		$remarks = "";
		$start = 0;
		$data = array();
		foreach ($artlist as $li) {
			$url = pq($li) -> find($pq_url) -> attr("href");
			//print_r($url);
			if ($this -> type == end(explode(".", $url))) {
				//print_r(pq($li)->text());
				$title = pq($li) -> find($pq_title) -> text();
				$remarks = pq($li) -> find($pq_remarks) -> text();
				$data[$start] = array(title => $title, url => $url, source => $source, remarks => $remarks);
				$start++;
			}
		}

		return $data;
	}

	/**创建格式化的url数组
	 * @param unknown_type $urls
	 * @return Ambigous <string, multitype:unknown >
	 */
	private function create_urls($urls) {
		$count = count($urls);
		$tmp = array();
		$m = 0;
		$page = 0;
		for ($m = 0; $m < $this -> page * $count; $m++) {
			$i = $m % $count;
			//echo $i;
			$page = floor($m / $count) * $urls[$i]['step'];
			$tmp[$m] = $urls[$i];
			if ($urls[$i]['keyrule'] == "+") {
				$keywords = str_replace(' ', '+', $this -> keywords);
			} else {
				$keywords = urlencode($this -> keywords);
			}
			$tmp[$m]['urls'] = str_replace("[KEYWORDS]", $keywords, $tmp[$m]['urls']);
			$tmp[$m]['urls'] = str_replace("[PAGE]", '', $tmp[$m]['urls']);
			$tmp[$m]['urls'] = $tmp[$m]['urls'] . $page;
		}
		return $tmp;
	}

	/**开始采集
	 * @return string
	 */
	public function start() {
		$this -> urls = $this -> create_urls($this -> urls);
		//print_r($this->urls);exit;
		if (!is_array($this -> urls) or count($this -> urls) == 0) {
			return false;
		}
		$curl = $text = array();
		$handle = curl_multi_init();
		foreach ($this->urls as $k => $v) {
			//echo $v['urls'].'<br/>';
			$curl[$k] = $this -> add_handle($handle, $v['urls']);
		}
		$this -> exec_handle($handle);
		foreach ($this->urls as $k => $v) {
			$text[$k] = curl_multi_getcontent($curl[$k]);
			if (!$this -> isUTF8($text[$k])) {
				$text[$k] = mb_convert_encoding($text[$k] . "\n\n", "UTF-8", "GBK");
			} else {
				$text[$k] = $text[$k] . "\n\n";
			}
			curl_multi_remove_handle($handle, $curl[$k]);
		}
		curl_multi_close($handle);
		return $text;
	}

	/** 数组去重
	 * @param unknown_type $array2D
	 * @param unknown_type $stkeep
	 * @param unknown_type $ndformat
	 * @return multitype:
	 */
	function unique_arr($array2D, $stkeep = false, $ndformat = true) {
		if (!empty($array2D)) {
			// 判断是否保留一级数组键 (一级数组键可以为非数字)
			if ($stkeep)
				$stArr = array_keys($array2D);

			// 判断是否保留二级数组键 (所有二级数组键必须相同)
			if ($ndformat)
				$ndArr = array_keys(end($array2D));

			//降维,也可以用implode,将一维数组转换为用逗号连接的字符串
			foreach ($array2D as $v) {
				$v = join("|smart|", $v);
				$temp[] = $v;
			}

			//去掉重复的字符串,也就是重复的一维数组
			$temp = array_unique($temp);

			//再将拆开的数组重新组装
			foreach ($temp as $k => $v) {
				if ($stkeep)
					$k = $stArr[$k];
				if ($ndformat) {
					$tempArr = explode("|smart|", $v);
					foreach ($tempArr as $ndkey => $ndval)
						$output[$k][$ndArr[$ndkey]] = $ndval;
				} else
					$output[$k] = explode(",", $v);
			}

			return $output;
		}
	}

	private function add_handle($handle, $url) {
		$curl = curl_init();
		$cookie_file = dirname(__FILE__) . "/temp/google.txt";
		//存储cookie值
		$headerArr = $this -> new_ip();
		//  print_r($headerArr);exit;
		$ch = curl_init();
		curl_setopt($curl, CURLOPT_URL, $url);
		curl_setopt($curl, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
		curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_HTTPHEADER, $headerArr);
		//构造IP
		// curl_setopt ($ch, CURLOPT_REFERER, "http://www.163.com/ ");   //构造来路
		curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1);
		curl_setopt($curl, CURLOPT_COOKIEJAR, $cookie_file);
		curl_multi_add_handle($handle, $curl);
		return $curl;
	}

	/**构建新ip
	 * @return ip;
	 */
	private function new_ip() {
		$ip2id = round(rand(600000, 2550000) / 10000) . "." . round(rand(600000, 2550000) / 10000) . "." . round(rand(600000, 2550000) / 10000) . "." . round(rand(600000, 2550000) / 10000);
		$headers['CLIENT-IP'] = $ip2id;
		$headers['X-FORWARDED-FOR'] = $ip2id;
		$headerArr = array();
		foreach ($headers as $n => $v) {
			$headerArr[] = $n . ':' . $v;
		}
		return $headerArr;
	}

	private function exec_handle($handle) {
		$active = null;
		do {
			$mrc = curl_multi_exec($handle, $active);
			//当无数据时或请求暂停时，active=true
		} while ($mrc == CURLM_CALL_MULTI_PERFORM);//当正在接受数据时
		while ($active and $mrc == CURLM_OK) {//当无数据时或请求暂停时，active=true,为了减少cpu的无谓负担,这一步很难明啊
			if (curl_multi_select($handle) != -1) {
				do {
					$mrc = curl_multi_exec($handle, $active);
				} while ($mrc == CURLM_CALL_MULTI_PERFORM);
			}
		}
	}

	/**
	 * 检测数据编码 ...
	 * @param 字符串 $str
	 * @return boolean
	 */
	private function isUTF8($str) {
		if ($str === mb_convert_encoding(mb_convert_encoding($str, "UTF-32", "UTF-8"), "UTF-8", "UTF-32")) {
			return true;
		} else {
			return false;
		}
	}

}
?>
